package com.chance.cc.crawler.development.bootstrap.gome;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * author zz
 * datetime 2021 11 23
  */
public class GomeStart {
    private static final String DOMAIN = "gome";
    private static final String SITE = "commodity";//
    private static final String START_URL = "https://search.gome.com.cn/search?question=nike&searchType=goods&search_mode=normal&reWrite=true&instock=1";


    public static void main(String[] args) {
        CrawlerRequestRecord beforeRecord = CrawlerRequestRecord.builder()
                .startPageRequest("gome_keyword", CrawlerEnum.CrawlerRequestType.turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/"+DOMAIN+"/keys?site=SearchKV")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(DOMAIN)
                .recordKey(START_URL)
                .httpUrl(START_URL)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(DOMAIN))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                //.resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .build();
        crawlerRequestRecord.getHttpRequest().addExtra("searchKvSourceUrl","https://search.gome.com.cn/search?question=%s&searchType=goods&search_mode=normal&reWrite=true&instock=1");
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(DOMAIN);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(SITE);
        //crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("product_url","product_url");//控制采集商品链接

        CrawlerRecord commentFilterRecord = new CrawlerRecord();
        commentFilterRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        commentFilterRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null));
        commentFilterRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,DOMAIN,"comment")));
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(commentFilterRecord));

        DevCrawlerController.builder()
                .crawlerThreadNum(10)
                .triggerInfo(DOMAIN,DOMAIN,System.currentTimeMillis(),DOMAIN)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(DOMAIN))
                .fileResultPipeline("F:\\changxi\\web_data\\nick.log",false)
                .consoleResultPipeline()
                .requestRecord(crawlerRequestRecord)
                .supportRecord(beforeRecord)
                .build("com.chance.cc.crawler.development.scripts.gome.GomeCrawlerScript")
                .start();

    }
}
